{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hzy/anaconda3/anaconda/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "/Users/hzy/anaconda3/anaconda/lib/python3.6/importlib/_bootstrap.py:205: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import xgboost\n",
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import tensorflow.contrib.eager as tfe\n",
    "import functools\n",
    "from sklearn.metrics import r2_score\n",
    "from src.utils import mape_score\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import Ridge, Lasso\n",
    "from sklearn.svm import SVR\n",
    "import pandas as pd\n",
    "tfe.enable_eager_execution()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('./data/sourceAB_shanghai_mall_processedB.csv', index_col=0)\n",
    "df_raw = pd.read_csv('./data/sourceAB.csv', index_col=0)\n",
    "\n",
    "Xa = df_raw.filter(regex='f_a.*', axis=1).values\n",
    "Xb = df_raw.filter(regex='f_b.*', axis=1).values\n",
    "y = np.asarray(df_raw['hotness'].values, dtype=np.float64)\n",
    "y = np.log(y)\n",
    "\n",
    "\n",
    "Xa = StandardScaler().fit_transform(Xa)\n",
    "Xb = StandardScaler().fit_transform(Xb)\n",
    "Xab = np.concatenate((Xa, Xb), axis=1)\n",
    "y = (y - y.mean() / y.std())\n",
    "\n",
    "numpy_all = {\n",
    "    'feature_a': Xa, \n",
    "    'feature_b': Xb, \n",
    "    'feature_ab': Xab,\n",
    "    'target': y\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "max_step = 3000\n",
    "feature_key = 'feature_ab'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from src.utils import validation_tf, kfold_tf, kfold_validation\n",
    "from src.tf_models import TypicalNetwork"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "use_feature: feature_ab\n",
      "fold: 1 / 4  r2: 0.519445 1-mape: -0.204229\n",
      "use_feature: feature_ab\n",
      "fold: 2 / 4  r2: 0.658937 1-mape: 0.325663\n",
      "use_feature: feature_ab\n",
      "fold: 3 / 4  r2: 0.568656 1-mape: -0.615631\n",
      "use_feature: feature_ab\n",
      "fold: 4 / 4  r2: 0.600864 1-mape: -0.413373\n",
      "avg score r2: 0.586975 1-mape: -0.226892\n"
     ]
    }
   ],
   "source": [
    "use_model = TypicalNetwork\n",
    "model_para = dict(feature_key=feature_key, hidden_size=40)\n",
    "steps, r2s, oneminus_mapes = kfold_tf(Xa, Xb, Xab, y, 4, use_model, model_para, batch_size, max_step, verbose=False)\n",
    "df = pd.DataFrame({'step': steps, 'r2':r2s, '1-mape': oneminus_mapes})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### normal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.53163595695570121"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kfold_validation(\n",
    "    numpy_all[feature_key],\n",
    "    numpy_all['target'],\n",
    "    4,\n",
    "    LinearRegression(),\n",
    "    use_mape=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.53270879308380226"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kfold_validation(\n",
    "    numpy_all[feature_key],\n",
    "    numpy_all['target'],\n",
    "    4,\n",
    "    Lasso(alpha=0.01),\n",
    "    use_mape=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.53239153336270306"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kfold_validation(\n",
    "    numpy_all[feature_key],\n",
    "    numpy_all['target'],\n",
    "    4,\n",
    "    Ridge(alpha=0.2),\n",
    "    use_mape=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.60970420085473853"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kfold_validation(\n",
    "    numpy_all[feature_key],\n",
    "    numpy_all['target'],\n",
    "    4,\n",
    "    xgboost.XGBRegressor(),\n",
    "    use_mape=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.56105418367054583"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kfold_validation(\n",
    "    numpy_all[feature_key],\n",
    "    numpy_all['target'],\n",
    "    4,\n",
    "    SVR(C=2.0),\n",
    "    use_mape=False\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2 nets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from src.tf_models import ProposedTwoLayers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "max_step = 5000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "use_model = ProposedTwoLayers\n",
    "model_para = dict(hidden_a = 20, hidden_b = 20, use_dropout = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "use_feature: feature_ab\n",
      "fold: 1 / 4  r2: 0.617446 1-mape: -0.313323\n",
      "use_feature: feature_ab\n",
      "fold: 2 / 4  r2: 0.637502 1-mape: -0.675510\n",
      "use_feature: feature_ab\n",
      "fold: 3 / 4  r2: 0.697999 1-mape: -0.112922\n",
      "use_feature: feature_ab\n",
      "fold: 4 / 4  r2: 0.503409 1-mape: -0.540979\n",
      "avg score r2: 0.614089 1-mape: -0.410683\n"
     ]
    }
   ],
   "source": [
    "steps, r2s, oneminus_mapes = kfold_tf(Xa, Xb, Xab, y, 4, use_model, model_para, batch_size, max_step, verbose=False)\n",
    "df = pd.DataFrame({'step': steps, 'r2':r2s, '1-mape': oneminus_mapes})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1-mape</th>\n",
       "      <th>r2</th>\n",
       "      <th>step</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>-0.426681</td>\n",
       "      <td>0.616396</td>\n",
       "      <td>4900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>-0.410683</td>\n",
       "      <td>0.614089</td>\n",
       "      <td>5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>-0.410683</td>\n",
       "      <td>0.614089</td>\n",
       "      <td>5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>-0.441644</td>\n",
       "      <td>0.613067</td>\n",
       "      <td>4400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>-0.427763</td>\n",
       "      <td>0.612812</td>\n",
       "      <td>4300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>-0.448097</td>\n",
       "      <td>0.612024</td>\n",
       "      <td>4800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>-0.424061</td>\n",
       "      <td>0.610406</td>\n",
       "      <td>4200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>-0.444603</td>\n",
       "      <td>0.610194</td>\n",
       "      <td>3900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>-0.418862</td>\n",
       "      <td>0.610172</td>\n",
       "      <td>4500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>-0.415775</td>\n",
       "      <td>0.609722</td>\n",
       "      <td>4600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>-0.451955</td>\n",
       "      <td>0.609525</td>\n",
       "      <td>4100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>-0.433894</td>\n",
       "      <td>0.609426</td>\n",
       "      <td>4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>-0.424370</td>\n",
       "      <td>0.609170</td>\n",
       "      <td>3800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>-0.428550</td>\n",
       "      <td>0.608449</td>\n",
       "      <td>3500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>-0.470961</td>\n",
       "      <td>0.608279</td>\n",
       "      <td>4700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>-0.457099</td>\n",
       "      <td>0.607347</td>\n",
       "      <td>3600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>-0.456921</td>\n",
       "      <td>0.605486</td>\n",
       "      <td>3700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>-0.466217</td>\n",
       "      <td>0.603941</td>\n",
       "      <td>3300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>-0.445015</td>\n",
       "      <td>0.603223</td>\n",
       "      <td>3100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>-0.439798</td>\n",
       "      <td>0.602897</td>\n",
       "      <td>3400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>-0.463588</td>\n",
       "      <td>0.601301</td>\n",
       "      <td>2900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>-0.470918</td>\n",
       "      <td>0.600970</td>\n",
       "      <td>3000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>-0.429937</td>\n",
       "      <td>0.600186</td>\n",
       "      <td>2800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>-0.437363</td>\n",
       "      <td>0.599944</td>\n",
       "      <td>2600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>-0.479579</td>\n",
       "      <td>0.599898</td>\n",
       "      <td>3200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>-0.444343</td>\n",
       "      <td>0.598914</td>\n",
       "      <td>2700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>-0.425492</td>\n",
       "      <td>0.595995</td>\n",
       "      <td>2300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>-0.464686</td>\n",
       "      <td>0.595760</td>\n",
       "      <td>2500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>-0.386866</td>\n",
       "      <td>0.593882</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>-0.410547</td>\n",
       "      <td>0.592728</td>\n",
       "      <td>2100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>-0.453743</td>\n",
       "      <td>0.591869</td>\n",
       "      <td>2400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>-0.451754</td>\n",
       "      <td>0.591731</td>\n",
       "      <td>2200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>-0.406101</td>\n",
       "      <td>0.589123</td>\n",
       "      <td>1800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>-0.395790</td>\n",
       "      <td>0.585776</td>\n",
       "      <td>1700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>-0.444038</td>\n",
       "      <td>0.584988</td>\n",
       "      <td>1900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>-0.408814</td>\n",
       "      <td>0.583639</td>\n",
       "      <td>1500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>-0.429113</td>\n",
       "      <td>0.582093</td>\n",
       "      <td>1600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>-0.419097</td>\n",
       "      <td>0.580481</td>\n",
       "      <td>1400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>-0.382657</td>\n",
       "      <td>0.577976</td>\n",
       "      <td>1300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>-0.394293</td>\n",
       "      <td>0.573521</td>\n",
       "      <td>1200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>-0.375807</td>\n",
       "      <td>0.572550</td>\n",
       "      <td>1100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>-0.369149</td>\n",
       "      <td>0.570745</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>-0.352353</td>\n",
       "      <td>0.569971</td>\n",
       "      <td>900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>-0.332004</td>\n",
       "      <td>0.565206</td>\n",
       "      <td>800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>-0.297395</td>\n",
       "      <td>0.542902</td>\n",
       "      <td>700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-0.285003</td>\n",
       "      <td>0.521739</td>\n",
       "      <td>600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.299654</td>\n",
       "      <td>0.505914</td>\n",
       "      <td>500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.386033</td>\n",
       "      <td>0.479317</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.484530</td>\n",
       "      <td>0.444735</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.582925</td>\n",
       "      <td>0.389972</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.667206</td>\n",
       "      <td>0.264845</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      1-mape        r2  step\n",
       "48 -0.426681  0.616396  4900\n",
       "50 -0.410683  0.614089  5000\n",
       "49 -0.410683  0.614089  5000\n",
       "43 -0.441644  0.613067  4400\n",
       "42 -0.427763  0.612812  4300\n",
       "47 -0.448097  0.612024  4800\n",
       "41 -0.424061  0.610406  4200\n",
       "38 -0.444603  0.610194  3900\n",
       "44 -0.418862  0.610172  4500\n",
       "45 -0.415775  0.609722  4600\n",
       "40 -0.451955  0.609525  4100\n",
       "39 -0.433894  0.609426  4000\n",
       "37 -0.424370  0.609170  3800\n",
       "34 -0.428550  0.608449  3500\n",
       "46 -0.470961  0.608279  4700\n",
       "35 -0.457099  0.607347  3600\n",
       "36 -0.456921  0.605486  3700\n",
       "32 -0.466217  0.603941  3300\n",
       "30 -0.445015  0.603223  3100\n",
       "33 -0.439798  0.602897  3400\n",
       "28 -0.463588  0.601301  2900\n",
       "29 -0.470918  0.600970  3000\n",
       "27 -0.429937  0.600186  2800\n",
       "25 -0.437363  0.599944  2600\n",
       "31 -0.479579  0.599898  3200\n",
       "26 -0.444343  0.598914  2700\n",
       "22 -0.425492  0.595995  2300\n",
       "24 -0.464686  0.595760  2500\n",
       "19 -0.386866  0.593882  2000\n",
       "20 -0.410547  0.592728  2100\n",
       "23 -0.453743  0.591869  2400\n",
       "21 -0.451754  0.591731  2200\n",
       "17 -0.406101  0.589123  1800\n",
       "16 -0.395790  0.585776  1700\n",
       "18 -0.444038  0.584988  1900\n",
       "14 -0.408814  0.583639  1500\n",
       "15 -0.429113  0.582093  1600\n",
       "13 -0.419097  0.580481  1400\n",
       "12 -0.382657  0.577976  1300\n",
       "11 -0.394293  0.573521  1200\n",
       "10 -0.375807  0.572550  1100\n",
       "9  -0.369149  0.570745  1000\n",
       "8  -0.352353  0.569971   900\n",
       "7  -0.332004  0.565206   800\n",
       "6  -0.297395  0.542902   700\n",
       "5  -0.285003  0.521739   600\n",
       "4  -0.299654  0.505914   500\n",
       "3  -0.386033  0.479317   400\n",
       "2  -0.484530  0.444735   300\n",
       "1  -0.582925  0.389972   200\n",
       "0  -0.667206  0.264845   100"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sort_values(\"r2\", ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plus out 2 Nets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from src.tf_models import ProposedTwoLayersPlusOut"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "max_step = 5000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "use_model = ProposedTwoLayersPlusOut\n",
    "model_para = dict(hidden_a = 20, hidden_b = 20, use_dropout = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "use_feature: feature_ab\n",
      "fold: 1 / 4  r2: 0.539037 1-mape: 0.144485\n",
      "use_feature: feature_ab\n",
      "fold: 2 / 4  r2: 0.624033 1-mape: 0.278411\n",
      "use_feature: feature_ab\n",
      "fold: 3 / 4  r2: 0.604089 1-mape: -0.964456\n",
      "use_feature: feature_ab\n",
      "fold: 4 / 4  r2: 0.730178 1-mape: -1.097735\n",
      "avg score r2: 0.624334 1-mape: -0.409824\n"
     ]
    }
   ],
   "source": [
    "steps, r2s, oneminus_mapes = kfold_tf(Xa, Xb, Xab, y, 4, use_model, model_para, batch_size, max_step, verbose=False)\n",
    "df = pd.DataFrame({'step': steps, 'r2':r2s, '1-mape': oneminus_mapes})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1-mape</th>\n",
       "      <th>r2</th>\n",
       "      <th>step</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>-0.311634</td>\n",
       "      <td>0.634039</td>\n",
       "      <td>4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>-0.282015</td>\n",
       "      <td>0.632800</td>\n",
       "      <td>4100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>-0.321139</td>\n",
       "      <td>0.631907</td>\n",
       "      <td>4600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>-0.306757</td>\n",
       "      <td>0.631630</td>\n",
       "      <td>4700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>-0.341180</td>\n",
       "      <td>0.631409</td>\n",
       "      <td>3900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>-0.320012</td>\n",
       "      <td>0.630593</td>\n",
       "      <td>3800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>-0.319151</td>\n",
       "      <td>0.629861</td>\n",
       "      <td>4400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>-0.267208</td>\n",
       "      <td>0.627441</td>\n",
       "      <td>4800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>-0.297638</td>\n",
       "      <td>0.627303</td>\n",
       "      <td>4200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>-0.323289</td>\n",
       "      <td>0.625599</td>\n",
       "      <td>3700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>-0.340879</td>\n",
       "      <td>0.624927</td>\n",
       "      <td>3600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>-0.358612</td>\n",
       "      <td>0.624810</td>\n",
       "      <td>4900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>-0.357785</td>\n",
       "      <td>0.624693</td>\n",
       "      <td>3500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>-0.382072</td>\n",
       "      <td>0.624529</td>\n",
       "      <td>4500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>-0.409824</td>\n",
       "      <td>0.624334</td>\n",
       "      <td>5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>-0.409824</td>\n",
       "      <td>0.624334</td>\n",
       "      <td>5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>-0.255891</td>\n",
       "      <td>0.623426</td>\n",
       "      <td>3400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>-0.316783</td>\n",
       "      <td>0.621057</td>\n",
       "      <td>3300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>-0.260948</td>\n",
       "      <td>0.620022</td>\n",
       "      <td>4300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>-0.312351</td>\n",
       "      <td>0.619902</td>\n",
       "      <td>3100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>-0.320424</td>\n",
       "      <td>0.619681</td>\n",
       "      <td>3200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>-0.333053</td>\n",
       "      <td>0.618662</td>\n",
       "      <td>2900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>-0.324965</td>\n",
       "      <td>0.615358</td>\n",
       "      <td>2800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>-0.416566</td>\n",
       "      <td>0.614599</td>\n",
       "      <td>3000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>-0.370836</td>\n",
       "      <td>0.611964</td>\n",
       "      <td>2700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>-0.316167</td>\n",
       "      <td>0.610180</td>\n",
       "      <td>2600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>-0.357171</td>\n",
       "      <td>0.606944</td>\n",
       "      <td>2300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>-0.326706</td>\n",
       "      <td>0.606640</td>\n",
       "      <td>2200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>-0.349967</td>\n",
       "      <td>0.604620</td>\n",
       "      <td>2400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>-0.287048</td>\n",
       "      <td>0.603914</td>\n",
       "      <td>2500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>-0.305826</td>\n",
       "      <td>0.603880</td>\n",
       "      <td>2100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>-0.339028</td>\n",
       "      <td>0.603143</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>-0.301791</td>\n",
       "      <td>0.600005</td>\n",
       "      <td>1900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>-0.366558</td>\n",
       "      <td>0.598955</td>\n",
       "      <td>1800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>-0.324565</td>\n",
       "      <td>0.593397</td>\n",
       "      <td>1700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>-0.317581</td>\n",
       "      <td>0.593312</td>\n",
       "      <td>1600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>-0.295515</td>\n",
       "      <td>0.591128</td>\n",
       "      <td>1500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>-0.273399</td>\n",
       "      <td>0.585673</td>\n",
       "      <td>1400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>-0.310849</td>\n",
       "      <td>0.584875</td>\n",
       "      <td>1300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>-0.254154</td>\n",
       "      <td>0.583677</td>\n",
       "      <td>1100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>-0.312381</td>\n",
       "      <td>0.583671</td>\n",
       "      <td>1200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>-0.263992</td>\n",
       "      <td>0.577064</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>-0.231069</td>\n",
       "      <td>0.575759</td>\n",
       "      <td>900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>-0.216563</td>\n",
       "      <td>0.569808</td>\n",
       "      <td>800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>-0.219417</td>\n",
       "      <td>0.565744</td>\n",
       "      <td>700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-0.224979</td>\n",
       "      <td>0.560318</td>\n",
       "      <td>600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.208306</td>\n",
       "      <td>0.555398</td>\n",
       "      <td>500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.226625</td>\n",
       "      <td>0.547671</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.266220</td>\n",
       "      <td>0.534579</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.436585</td>\n",
       "      <td>0.501686</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.645849</td>\n",
       "      <td>0.376836</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      1-mape        r2  step\n",
       "39 -0.311634  0.634039  4000\n",
       "40 -0.282015  0.632800  4100\n",
       "45 -0.321139  0.631907  4600\n",
       "46 -0.306757  0.631630  4700\n",
       "38 -0.341180  0.631409  3900\n",
       "37 -0.320012  0.630593  3800\n",
       "43 -0.319151  0.629861  4400\n",
       "47 -0.267208  0.627441  4800\n",
       "41 -0.297638  0.627303  4200\n",
       "36 -0.323289  0.625599  3700\n",
       "35 -0.340879  0.624927  3600\n",
       "48 -0.358612  0.624810  4900\n",
       "34 -0.357785  0.624693  3500\n",
       "44 -0.382072  0.624529  4500\n",
       "50 -0.409824  0.624334  5000\n",
       "49 -0.409824  0.624334  5000\n",
       "33 -0.255891  0.623426  3400\n",
       "32 -0.316783  0.621057  3300\n",
       "42 -0.260948  0.620022  4300\n",
       "30 -0.312351  0.619902  3100\n",
       "31 -0.320424  0.619681  3200\n",
       "28 -0.333053  0.618662  2900\n",
       "27 -0.324965  0.615358  2800\n",
       "29 -0.416566  0.614599  3000\n",
       "26 -0.370836  0.611964  2700\n",
       "25 -0.316167  0.610180  2600\n",
       "22 -0.357171  0.606944  2300\n",
       "21 -0.326706  0.606640  2200\n",
       "23 -0.349967  0.604620  2400\n",
       "24 -0.287048  0.603914  2500\n",
       "20 -0.305826  0.603880  2100\n",
       "19 -0.339028  0.603143  2000\n",
       "18 -0.301791  0.600005  1900\n",
       "17 -0.366558  0.598955  1800\n",
       "16 -0.324565  0.593397  1700\n",
       "15 -0.317581  0.593312  1600\n",
       "14 -0.295515  0.591128  1500\n",
       "13 -0.273399  0.585673  1400\n",
       "12 -0.310849  0.584875  1300\n",
       "10 -0.254154  0.583677  1100\n",
       "11 -0.312381  0.583671  1200\n",
       "9  -0.263992  0.577064  1000\n",
       "8  -0.231069  0.575759   900\n",
       "7  -0.216563  0.569808   800\n",
       "6  -0.219417  0.565744   700\n",
       "5  -0.224979  0.560318   600\n",
       "4  -0.208306  0.555398   500\n",
       "3  -0.226625  0.547671   400\n",
       "2  -0.266220  0.534579   300\n",
       "1  -0.436585  0.501686   200\n",
       "0  -0.645849  0.376836   100"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sort_values(\"r2\", ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
